##%%%%%%%%%%%%%%%%%%%%%%% simOfDemingRREG %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##%%%%%%%%%%%%%%% Ridge Regression of Deming Data %%%%%%%%%%%%%%%%%%%%%
## In this section we control overfitting of the original Deming
## fixed effects models using ridge regression.

library(Matrix)
library(MASS) ## For ginv() - Moore penrose inverse

##%%%%%%%% Set Working Directory %%%%%%%%%%%%
## !!! Here one needs to set one's own working directory
## where the script and data files are held !!!
WorkDir="D:/RWORK/Repository Files"
setwd(WorkDir)

source("FUNC.R")

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##%%%%%%%%%%%%%%%%%%%% Basic Calculations  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

## Reload the saved files
load("tins2")
load("tins7")


##%%%%%%%% Define II mats for following cv computations
## Here the in and out data are the same 
## !! ignore out/outL.stat's !!
iiin=1:1115
iiout=1:1115
iioutL=1:1115

rsqMt17=makeRsqMat0.1(tins7[,1],tins7[,3:ncol(tins7)],iiin,iiout,iioutL)
rsqMt27=makeRsqMat0.1(tins7[,2],tins7[,3:ncol(tins7)],iiin,iiout,iioutL)

## Merge these
rsqNamesN1=c("rsqMt12","rsqMt22","rsqMt13","rsqMt23","rsqMt14","rsqMt24",
		"rsqMt14S","rsqMt24S","rsqMt17S","rsqMt27S","rsqMt17","rsqMt27")

origNamesN1=c("xx2t1N1","xx2t2N1","xx3t1N1","xx3t2N1","xx4t14N1","xx4t2N1",
		"xx4St1N1","xx4St2N1","xx7St1N1","xx7St2N1","xx7t1N1","xx7t2N1")

## Get column sets
rsqCols=colnames(rsqMt14)[1:59]
rsqColsNoOut=rsqCols[substring(rsqCols,1,3)!="out"]
jjModStats=c(1:6,9:12)

## Merge
foo=get(rsqNamesN1[1])
foo1=foo[1,jjModStats]
rownames(foo)=paste(origNamesN1[1],rownames(foo))
rsqOrigN1=foo[,rsqColsNoOut]
modStatsOrigN1=foo1


for(i in 2:length(rsqNamesN1)){
	foo=get(rsqNamesN1[i])
	foo1=foo[1,jjModStats]
	rownames(foo)=paste(origNamesN1[i],rownames(foo))
	rsqOrigN1=rbind(rsqOrigN1,foo[,rsqColsNoOut])
	modStatsOrigN1=rbind(modStatsOrigN1,foo1)
	}
rownames(modStatsOrigN1)=origNamesN1





## Do the same thing for "male" subset
malEin=malE
malEout=malE
malEoutL=malE
rsqMt17m=makeRsqMat0.1(tins7[,1],tins7[,3:ncol(tins7)],malEin,malEout,malEoutL)
rsqMt27m=makeRsqMat0.1(tins7[,2],tins7[,3:ncol(tins7)],malEin,malEout,malEoutL)

##%%%%%%%%%% NOTE!!! 
## The results for the "male" subset here don't exactly match that
## found in Deming09 because we have restricted the subset of inputs
## to those that weren't dropped in the original fit to t1 or t2
## on the full data set of 1115.  The results change somewhat if 
## one leaves all these in and only drops those that are redundant 
## for each subsequent fit on the reduced data set. 
source("D:/Desco/Splus/clusBootCVdv.ssc")
load("D:/RWORK/clusFEsfe")

## Cluster index matrices
clusMM=clusFEsfe

## Make a cluster boot index for N=50
clusBootI=clusBootIndex(clusM=clusMM,N=50)

## Identify the HS variable for drop
dV=colnames(cov7)[1]
dvI=1

## cluster boot cross validation
clList1=clusBootIndexLoocvOLSdv(t1,cov7,clusMM,clusBootI,dV,dvI)




